import cPickle
import sys
import os
from collections import Counter
from collections import defaultdict
from operator import itemgetter
import numpy as np 
from numpy import arange,array,ones,linalg,bincount

xindex={}
c1_db="clus_1.db"
c2_db="clus_2.db"
cno_db="clus_no.db"
for x in open("xindex.db"):
	xindex[x.strip("\n")]=1

tm_index=sorted(xindex)

def init_bl():
	return {}

def init_db(dbn,blacklist):
	db=cPickle.load(open(dbn))
	for k in db.keys():
		if k in blacklist:
			db.pop(k)
	return db

def cut_data():
	db=init_db(cno_db,init_bl())
	out={}
	tmp={}
	tmp_no={}
	blacklist={}
	for k in db:
		if not k[0] and not k[1] and not k[2] and not k[3] and not k[4]:
			blacklist[k]=1
			blacklist[db[k]]=1
			continue
		if k[0]:
			continue
		if k[2]:
			continue
		out[k]=db[k]

	for k in out:
		rk=k[:5]
		c=k[5]
		if rk in tmp:
			tmp[rk]+=1
			tmp_no[rk].append((k,out[k]))
		else:
			tmp[rk]=1
			tmp_no[rk]=[]
			tmp_no[rk].append((k,out[k]))
	
	t=sorted( v[1] for v in tmp.items())
	cp=int(float(len(t))*0.99)
	val=t[cp]
	if val<10:
		val=10
	t=[v for v in t if v>val]
	for k in tmp:
		if tmp[k] in t:
			for tk in tmp_no[k]:
				blacklist[tk[0]]=1
				blacklist[tk[1]]=1
	print len(blacklist)
	f=open("bl.db","w")
	cPickle.dump(blacklist,f)
	f.close()




if __name__ == '__main__':
	cut_data()